import glob
import json
from tqdm import tqdm

input_files = glob.glob('../data/raw_data/action_response_data/*.json')


def longestCommonSubsequence(text1, text2, percent=0.7):
    m, n = len(text1), len(text2)
    dp = [[0] * (n + 1) for _ in range(m + 1)]

    for i in range(1, m + 1):
        for j in range(1, n + 1):
            if text1[i - 1] == text2[j - 1]:
                dp[i][j] = dp[i - 1][j - 1] + 1
            else:
                dp[i][j] = max(dp[i][j - 1], dp[i - 1][j])

    repetition_len = dp[-1][-1]
    if repetition_len / m >= percent or repetition_len / n >= percent:
        return True
    else:
        return False


all_repetition_positions = []

for f in tqdm(input_files):
    session_id = f.split('/')[-1].replace('.json', '')

    corpus = json.load(open(f, 'r', encoding='utf-8'))
    dialog = corpus['dialog']

    history_response = []
    for i, turn in enumerate(dialog):
        no_modified_bot_response = turn['bot_response']
        modified_bot_response = "".join([t['content'] for t in turn['new_outputs']])

        turn_id = turn['turn_id']
        if history_response:
            if turn['skip'] == 1:
                continue

            if modified_bot_response == '':
                continue

            for j, hisr in enumerate(reversed(history_response[-5:])):
                if '还有' in hisr or '还有' in modified_bot_response:
                    continue

                if longestCommonSubsequence(hisr, modified_bot_response):
                    repetition_dict = {
                        'session_id': session_id,
                        'prev_turn_id': int(turn_id) - j - 1,
                        'cur_turn_id': turn_id,
                        'past_utter': hisr,
                        # 'current_utter': no_modified_bot_response,
                        'modified_utter': modified_bot_response,
                        'new_modified': ''
                    }

                    all_repetition_positions.append(repetition_dict)

        history_response.append(no_modified_bot_response)

print(len(all_repetition_positions))
json.dump(all_repetition_positions,
          open('repetition_position.json', 'w', encoding='utf-8'), ensure_ascii=False)
